__author__ = 'Py.ziMing'
import json
import os
import re
from hashlib import md5
from multiprocessing.pool import Pool
from urllib.parse import urlencode

import requests
from bs4 import BeautifulSoup
from requests import codes, RequestException

offset = '0'
keyword = '街拍'

GROUP_START = 1
GROUP_END = 20

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64)",-
    "Referer": "https://www.toutiao.com/"
}


# 获取索引页：
def get_index_page(offset, keyword):
    params = {
        'offset': offset,
        'format': 'json',
        'keyword': keyword,
        'autoload': 'true',
        'count': '20',
        'cur_tab': '1',
        'from': 'search_tab',
        'pd': 'synthesis'
    }
    base_url = 'https://www.toutiao.com/api/search/content/?'
    url = base_url + urlencode(params)
    try:
        response = requests.get(url=url, headers=headers)
        if response.status_code == codes.ok:
            print(url + ':导航页请求成功！')
            return response.text
    except requests.ConnectionError:
        print('请求导航页失败！')
        return None


# 解析索引页内容
# 分析ajax请求的返回结果，拿到详情页的url
def parse_index_page(html):
    data = json.loads(html)
    if data and 'data' in data.keys():
        for item in data.get('data'):
            if item.get('cell_type') is not None:  # 去除没有url的item
                continue
            yield item.get('article_url')


# 根据索引页的url获取详情页
def get_detail_page(url):
    try:
        response = requests.get(url=url, headers=headers)
        if response.status_code == 200:
            print(url + ':详情页请求成功！')
            return response.text
        else:
            return None
    except RequestException:
        print('请求详情页错误', url)
        return None


# 解析详情页的内容
def parse_detail_page(html, url):
    try:
        soup = BeautifulSoup(html, 'lxml')
        title = soup.select('title')[0].get_text()  # 获取文章title
        print(title)
    #     images_pattern_1 = re.compile('gallery: JSON.parse\("(.*?)"\)', re.S)  # 匹配模式
    #     result_1 = re.search(images_pattern_1, html)  # 匹配内容
    #     images_pattern_2 = re.compile('img src&#x3D;&quot;(.*?)&quot;', re.S)
    #     result_2 = re.findall(images_pattern_2, html)  # 匹配内容
    #     if result_1:  # 第一种网页：点击才能跳转图片
    #         str = re.sub(r'(\\)', '', result_1.group(1))  # 去掉url链接中多余的双斜线“\\”
    #         if str:  # 如果匹配到内容，执行接下来的操作
    #             data = json.loads(str)
    #             if data and 'sub_images' in data.keys():  # 确保返回的信息中含有sub_images这个信息
    #                 sub_images = data.get('sub_images')
    #                 images = [item.get('url') for item in sub_images]  # 提取sub_images中图片的url链接
    #                 yield {
    #                     'title': title,  # 详情页标题
    #                     'url': url,  # 详情页链接
    #                     'images': images  # 图片链接
    #                 }
    #     elif result_2:  # 第二种网页： 往下滑动就能看到全部图片
    #         # reulut_2返回的就是一个列表
    #         yield {
    #             'title': title,  # 详情页标题
    #             'url': url,  # 详情页链接
    #             'images': result_2  # 图片链接
    #         }
    except:
        return None  # 跳过异常继续执行


# 下载图片（下载+存图）
# def download_img(image_url):
#     print('正在下载:', image_url)
#     try:
#         response = requests.get(image_url, headers=headers)
#         if response.status_code == 200:
#             save_to_local_file(response.content)  # content返回二进制内容，图片一般返回content
#         else:
#             return None
#     except RequestException:
#         print('下载图片错误:', image_url)
#         return None


# 存储图片方法
# def save_to_local_file(content):
#     img_path = 'img'
#     if not os.path.exists(img_path):
#         os.makedirs(img_path)
#     file_path = img_path + os.path.sep + '{file_name}.{file_suffix}'.format(
#         file_name=md5(content).hexdigest(),
#         file_suffix='jpg')
#     with open(file_path, 'wb') as f:
#         f.write(content)


# 定义主函数，调用之前的方法
def main(offset):
    html = get_index_page(offset=offset, keyword=keyword)
    for url in parse_index_page(html):  # 返回的是一个迭代器，每次输出一个网址
        html = get_detail_page(url)
        if html:
            result = parse_detail_page(html, url)  # 传入详情页链接、详情页内容，进行解析
            print(result)
            # for item in parse_detail_page(html, url):
            #     image_list = item.get('images')  # 传入图片链接，下载并保存到本地
            #     for image in image_list:
            #         download_img(image)


# 只运行本文件中的主函数
if __name__ == '__main__':
    groups = [i * 20 for i in list(range(GROUP_START, GROUP_END))]  # python3 range()不能直接生成列表，需要list一下
    pool = Pool()  # 创建进程池
    pool.map(main, groups)  # 第一个参数是函数，第二个参数是一个迭代器，将迭代器中的数字作为参数依次传入函数中
    pool.close()
    pool.join()